{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "popular-swimming",
   "metadata": {},
   "source": [
    "## 单特征筛选"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "talented-amount",
   "metadata": {},
   "source": [
    "- toad库来过滤大量的特征，高缺失率、低iv和高度相关的特征一次性过滤掉"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "charged-czech",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 21)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import toad\n",
    "\n",
    "data = pd.read_csv('data/germancredit.csv')\n",
    "data.replace({'good':0,'bad':1},inplace=True)\n",
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "planned-ballot",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>status.of.existing.checking.account</th>\n",
       "      <th>duration.in.month</th>\n",
       "      <th>credit.history</th>\n",
       "      <th>purpose</th>\n",
       "      <th>credit.amount</th>\n",
       "      <th>savings.account.and.bonds</th>\n",
       "      <th>present.employment.since</th>\n",
       "      <th>installment.rate.in.percentage.of.disposable.income</th>\n",
       "      <th>personal.status.and.sex</th>\n",
       "      <th>other.debtors.or.guarantors</th>\n",
       "      <th>...</th>\n",
       "      <th>property</th>\n",
       "      <th>age.in.years</th>\n",
       "      <th>other.installment.plans</th>\n",
       "      <th>housing</th>\n",
       "      <th>number.of.existing.credits.at.this.bank</th>\n",
       "      <th>job</th>\n",
       "      <th>number.of.people.being.liable.to.provide.maintenance.for</th>\n",
       "      <th>telephone</th>\n",
       "      <th>foreign.worker</th>\n",
       "      <th>creditability</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>... &lt; 0 DM</td>\n",
       "      <td>6</td>\n",
       "      <td>critical account/ other credits existing (not ...</td>\n",
       "      <td>radio/television</td>\n",
       "      <td>1169</td>\n",
       "      <td>unknown/ no savings account</td>\n",
       "      <td>... &gt;= 7 years</td>\n",
       "      <td>4</td>\n",
       "      <td>male : single</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>real estate</td>\n",
       "      <td>67</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>2</td>\n",
       "      <td>skilled employee / official</td>\n",
       "      <td>1</td>\n",
       "      <td>yes, registered under the customers name</td>\n",
       "      <td>yes</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0 &lt;= ... &lt; 200 DM</td>\n",
       "      <td>48</td>\n",
       "      <td>existing credits paid back duly till now</td>\n",
       "      <td>radio/television</td>\n",
       "      <td>5951</td>\n",
       "      <td>... &lt; 100 DM</td>\n",
       "      <td>1 &lt;= ... &lt; 4 years</td>\n",
       "      <td>2</td>\n",
       "      <td>female : divorced/separated/married</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>real estate</td>\n",
       "      <td>22</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>1</td>\n",
       "      <td>skilled employee / official</td>\n",
       "      <td>1</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>no checking account</td>\n",
       "      <td>12</td>\n",
       "      <td>critical account/ other credits existing (not ...</td>\n",
       "      <td>education</td>\n",
       "      <td>2096</td>\n",
       "      <td>... &lt; 100 DM</td>\n",
       "      <td>4 &lt;= ... &lt; 7 years</td>\n",
       "      <td>2</td>\n",
       "      <td>male : single</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>real estate</td>\n",
       "      <td>49</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>1</td>\n",
       "      <td>unskilled - resident</td>\n",
       "      <td>2</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>... &lt; 0 DM</td>\n",
       "      <td>42</td>\n",
       "      <td>existing credits paid back duly till now</td>\n",
       "      <td>furniture/equipment</td>\n",
       "      <td>7882</td>\n",
       "      <td>... &lt; 100 DM</td>\n",
       "      <td>4 &lt;= ... &lt; 7 years</td>\n",
       "      <td>2</td>\n",
       "      <td>male : single</td>\n",
       "      <td>guarantor</td>\n",
       "      <td>...</td>\n",
       "      <td>building society savings agreement/ life insur...</td>\n",
       "      <td>45</td>\n",
       "      <td>none</td>\n",
       "      <td>for free</td>\n",
       "      <td>1</td>\n",
       "      <td>skilled employee / official</td>\n",
       "      <td>2</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>... &lt; 0 DM</td>\n",
       "      <td>24</td>\n",
       "      <td>delay in paying off in the past</td>\n",
       "      <td>car (new)</td>\n",
       "      <td>4870</td>\n",
       "      <td>... &lt; 100 DM</td>\n",
       "      <td>1 &lt;= ... &lt; 4 years</td>\n",
       "      <td>3</td>\n",
       "      <td>male : single</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>unknown / no property</td>\n",
       "      <td>53</td>\n",
       "      <td>none</td>\n",
       "      <td>for free</td>\n",
       "      <td>2</td>\n",
       "      <td>skilled employee / official</td>\n",
       "      <td>2</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>no checking account</td>\n",
       "      <td>12</td>\n",
       "      <td>existing credits paid back duly till now</td>\n",
       "      <td>furniture/equipment</td>\n",
       "      <td>1736</td>\n",
       "      <td>... &lt; 100 DM</td>\n",
       "      <td>4 &lt;= ... &lt; 7 years</td>\n",
       "      <td>3</td>\n",
       "      <td>female : divorced/separated/married</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>real estate</td>\n",
       "      <td>31</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>1</td>\n",
       "      <td>unskilled - resident</td>\n",
       "      <td>1</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>... &lt; 0 DM</td>\n",
       "      <td>30</td>\n",
       "      <td>existing credits paid back duly till now</td>\n",
       "      <td>car (used)</td>\n",
       "      <td>3857</td>\n",
       "      <td>... &lt; 100 DM</td>\n",
       "      <td>1 &lt;= ... &lt; 4 years</td>\n",
       "      <td>4</td>\n",
       "      <td>female : divorced/separated/married</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>building society savings agreement/ life insur...</td>\n",
       "      <td>40</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>1</td>\n",
       "      <td>management/ self-employed/ highly qualified em...</td>\n",
       "      <td>1</td>\n",
       "      <td>yes, registered under the customers name</td>\n",
       "      <td>yes</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>no checking account</td>\n",
       "      <td>12</td>\n",
       "      <td>existing credits paid back duly till now</td>\n",
       "      <td>radio/television</td>\n",
       "      <td>804</td>\n",
       "      <td>... &lt; 100 DM</td>\n",
       "      <td>... &gt;= 7 years</td>\n",
       "      <td>4</td>\n",
       "      <td>male : single</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>car or other, not in attribute Savings account...</td>\n",
       "      <td>38</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>1</td>\n",
       "      <td>skilled employee / official</td>\n",
       "      <td>1</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>... &lt; 0 DM</td>\n",
       "      <td>45</td>\n",
       "      <td>existing credits paid back duly till now</td>\n",
       "      <td>radio/television</td>\n",
       "      <td>1845</td>\n",
       "      <td>... &lt; 100 DM</td>\n",
       "      <td>1 &lt;= ... &lt; 4 years</td>\n",
       "      <td>4</td>\n",
       "      <td>male : single</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>unknown / no property</td>\n",
       "      <td>23</td>\n",
       "      <td>none</td>\n",
       "      <td>for free</td>\n",
       "      <td>1</td>\n",
       "      <td>skilled employee / official</td>\n",
       "      <td>1</td>\n",
       "      <td>yes, registered under the customers name</td>\n",
       "      <td>yes</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>0 &lt;= ... &lt; 200 DM</td>\n",
       "      <td>45</td>\n",
       "      <td>critical account/ other credits existing (not ...</td>\n",
       "      <td>car (used)</td>\n",
       "      <td>4576</td>\n",
       "      <td>100 &lt;= ... &lt; 500 DM</td>\n",
       "      <td>unemployed</td>\n",
       "      <td>3</td>\n",
       "      <td>male : single</td>\n",
       "      <td>none</td>\n",
       "      <td>...</td>\n",
       "      <td>car or other, not in attribute Savings account...</td>\n",
       "      <td>27</td>\n",
       "      <td>none</td>\n",
       "      <td>own</td>\n",
       "      <td>1</td>\n",
       "      <td>skilled employee / official</td>\n",
       "      <td>1</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    status.of.existing.checking.account  duration.in.month  \\\n",
       "0                            ... < 0 DM                  6   \n",
       "1                     0 <= ... < 200 DM                 48   \n",
       "2                   no checking account                 12   \n",
       "3                            ... < 0 DM                 42   \n",
       "4                            ... < 0 DM                 24   \n",
       "..                                  ...                ...   \n",
       "995                 no checking account                 12   \n",
       "996                          ... < 0 DM                 30   \n",
       "997                 no checking account                 12   \n",
       "998                          ... < 0 DM                 45   \n",
       "999                   0 <= ... < 200 DM                 45   \n",
       "\n",
       "                                        credit.history              purpose  \\\n",
       "0    critical account/ other credits existing (not ...     radio/television   \n",
       "1             existing credits paid back duly till now     radio/television   \n",
       "2    critical account/ other credits existing (not ...            education   \n",
       "3             existing credits paid back duly till now  furniture/equipment   \n",
       "4                      delay in paying off in the past            car (new)   \n",
       "..                                                 ...                  ...   \n",
       "995           existing credits paid back duly till now  furniture/equipment   \n",
       "996           existing credits paid back duly till now           car (used)   \n",
       "997           existing credits paid back duly till now     radio/television   \n",
       "998           existing credits paid back duly till now     radio/television   \n",
       "999  critical account/ other credits existing (not ...           car (used)   \n",
       "\n",
       "     credit.amount    savings.account.and.bonds present.employment.since  \\\n",
       "0             1169  unknown/ no savings account           ... >= 7 years   \n",
       "1             5951                 ... < 100 DM       1 <= ... < 4 years   \n",
       "2             2096                 ... < 100 DM       4 <= ... < 7 years   \n",
       "3             7882                 ... < 100 DM       4 <= ... < 7 years   \n",
       "4             4870                 ... < 100 DM       1 <= ... < 4 years   \n",
       "..             ...                          ...                      ...   \n",
       "995           1736                 ... < 100 DM       4 <= ... < 7 years   \n",
       "996           3857                 ... < 100 DM       1 <= ... < 4 years   \n",
       "997            804                 ... < 100 DM           ... >= 7 years   \n",
       "998           1845                 ... < 100 DM       1 <= ... < 4 years   \n",
       "999           4576          100 <= ... < 500 DM               unemployed   \n",
       "\n",
       "     installment.rate.in.percentage.of.disposable.income  \\\n",
       "0                                                    4     \n",
       "1                                                    2     \n",
       "2                                                    2     \n",
       "3                                                    2     \n",
       "4                                                    3     \n",
       "..                                                 ...     \n",
       "995                                                  3     \n",
       "996                                                  4     \n",
       "997                                                  4     \n",
       "998                                                  4     \n",
       "999                                                  3     \n",
       "\n",
       "                 personal.status.and.sex other.debtors.or.guarantors  ...  \\\n",
       "0                          male : single                        none  ...   \n",
       "1    female : divorced/separated/married                        none  ...   \n",
       "2                          male : single                        none  ...   \n",
       "3                          male : single                   guarantor  ...   \n",
       "4                          male : single                        none  ...   \n",
       "..                                   ...                         ...  ...   \n",
       "995  female : divorced/separated/married                        none  ...   \n",
       "996  female : divorced/separated/married                        none  ...   \n",
       "997                        male : single                        none  ...   \n",
       "998                        male : single                        none  ...   \n",
       "999                        male : single                        none  ...   \n",
       "\n",
       "                                              property age.in.years  \\\n",
       "0                                          real estate           67   \n",
       "1                                          real estate           22   \n",
       "2                                          real estate           49   \n",
       "3    building society savings agreement/ life insur...           45   \n",
       "4                                unknown / no property           53   \n",
       "..                                                 ...          ...   \n",
       "995                                        real estate           31   \n",
       "996  building society savings agreement/ life insur...           40   \n",
       "997  car or other, not in attribute Savings account...           38   \n",
       "998                              unknown / no property           23   \n",
       "999  car or other, not in attribute Savings account...           27   \n",
       "\n",
       "     other.installment.plans   housing  \\\n",
       "0                       none       own   \n",
       "1                       none       own   \n",
       "2                       none       own   \n",
       "3                       none  for free   \n",
       "4                       none  for free   \n",
       "..                       ...       ...   \n",
       "995                     none       own   \n",
       "996                     none       own   \n",
       "997                     none       own   \n",
       "998                     none  for free   \n",
       "999                     none       own   \n",
       "\n",
       "    number.of.existing.credits.at.this.bank  \\\n",
       "0                                         2   \n",
       "1                                         1   \n",
       "2                                         1   \n",
       "3                                         1   \n",
       "4                                         2   \n",
       "..                                      ...   \n",
       "995                                       1   \n",
       "996                                       1   \n",
       "997                                       1   \n",
       "998                                       1   \n",
       "999                                       1   \n",
       "\n",
       "                                                   job  \\\n",
       "0                          skilled employee / official   \n",
       "1                          skilled employee / official   \n",
       "2                                 unskilled - resident   \n",
       "3                          skilled employee / official   \n",
       "4                          skilled employee / official   \n",
       "..                                                 ...   \n",
       "995                               unskilled - resident   \n",
       "996  management/ self-employed/ highly qualified em...   \n",
       "997                        skilled employee / official   \n",
       "998                        skilled employee / official   \n",
       "999                        skilled employee / official   \n",
       "\n",
       "    number.of.people.being.liable.to.provide.maintenance.for  \\\n",
       "0                                                    1         \n",
       "1                                                    1         \n",
       "2                                                    2         \n",
       "3                                                    2         \n",
       "4                                                    2         \n",
       "..                                                 ...         \n",
       "995                                                  1         \n",
       "996                                                  1         \n",
       "997                                                  1         \n",
       "998                                                  1         \n",
       "999                                                  1         \n",
       "\n",
       "                                    telephone foreign.worker creditability  \n",
       "0    yes, registered under the customers name            yes             0  \n",
       "1                                        none            yes             1  \n",
       "2                                        none            yes             0  \n",
       "3                                        none            yes             0  \n",
       "4                                        none            yes             1  \n",
       "..                                        ...            ...           ...  \n",
       "995                                      none            yes             0  \n",
       "996  yes, registered under the customers name            yes             0  \n",
       "997                                      none            yes             0  \n",
       "998  yes, registered under the customers name            yes             1  \n",
       "999                                      none            yes             0  \n",
       "\n",
       "[1000 rows x 21 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "religious-colon",
   "metadata": {},
   "outputs": [],
   "source": [
    "#缺失率大于0.5,IV值小于0.05,相关性大于0.7来进行特征筛选\n",
    "selected_data, drop_list= toad.selection.select(data,target = 'creditability', empty = 0.5, iv = 0.05, corr = 0.7, return_drop=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "afraid-stanley",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'empty': array([], dtype=float64),\n",
       " 'iv': array(['installment.rate.in.percentage.of.disposable.income',\n",
       "        'personal.status.and.sex', 'other.debtors.or.guarantors',\n",
       "        'present.residence.since',\n",
       "        'number.of.existing.credits.at.this.bank', 'job',\n",
       "        'number.of.people.being.liable.to.provide.maintenance.for',\n",
       "        'telephone', 'foreign.worker'], dtype=object),\n",
       " 'corr': array([], dtype=object)}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "drop_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "enclosed-mount",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "保留特征: 12 缺失删除: 0 低iv删除： 9 高相关删除： 0\n"
     ]
    }
   ],
   "source": [
    "print('保留特征:',selected_data.shape[1],'缺失删除:',len(drop_list['empty']),'低iv删除：',len(drop_list['iv']),'高相关删除：',len(drop_list['corr']))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "forbidden-hardware",
   "metadata": {},
   "source": [
    "## 多特征筛选"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "featured-brand",
   "metadata": {},
   "source": [
    "- Boruta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "decent-slope",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: http://mirrors.aliyun.com/pypi/simple\n",
      "Requirement already satisfied: Boruta in /Users/vincent/anaconda3/envs/jlab/lib/python3.8/site-packages (0.3)\n",
      "Requirement already satisfied: scikit-learn>=0.17.1 in /Users/vincent/.local/lib/python3.8/site-packages (from Boruta) (0.24.1)\n",
      "Requirement already satisfied: numpy>=1.10.4 in /Users/vincent/anaconda3/envs/jlab/lib/python3.8/site-packages (from Boruta) (1.19.5)\n",
      "Requirement already satisfied: scipy>=0.17.0 in /Users/vincent/.local/lib/python3.8/site-packages (from Boruta) (1.6.1)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/vincent/anaconda3/envs/jlab/lib/python3.8/site-packages (from scikit-learn>=0.17.1->Boruta) (2.1.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /Users/vincent/.local/lib/python3.8/site-packages (from scikit-learn>=0.17.1->Boruta) (1.0.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install Boruta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "requested-recorder",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>AMT_GOODS_PRICE</th>\n",
       "      <th>REGION_POPULATION_RELATIVE</th>\n",
       "      <th>DAYS_BIRTH</th>\n",
       "      <th>DAYS_EMPLOYED</th>\n",
       "      <th>DAYS_REGISTRATION</th>\n",
       "      <th>DAYS_ID_PUBLISH</th>\n",
       "      <th>REGION_RATING_CLIENT_W_CITY</th>\n",
       "      <th>REG_CITY_NOT_LIVE_CITY</th>\n",
       "      <th>...</th>\n",
       "      <th>p_NAME_SELLER_INDUSTRY_Connectivity</th>\n",
       "      <th>p_NAME_YIELD_GROUP_XNA</th>\n",
       "      <th>p_NAME_YIELD_GROUP_high</th>\n",
       "      <th>p_NAME_YIELD_GROUP_low_action</th>\n",
       "      <th>p_NAME_YIELD_GROUP_low_normal</th>\n",
       "      <th>p_PRODUCT_COMBINATION_Card Street</th>\n",
       "      <th>p_PRODUCT_COMBINATION_Cash Street: high</th>\n",
       "      <th>p_PRODUCT_COMBINATION_Cash X-Sell: high</th>\n",
       "      <th>p_PRODUCT_COMBINATION_Cash X-Sell: low</th>\n",
       "      <th>p_PRODUCT_COMBINATION_POS industry with interest</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>125406</th>\n",
       "      <td>245429</td>\n",
       "      <td>0</td>\n",
       "      <td>0.610118</td>\n",
       "      <td>0.016406</td>\n",
       "      <td>0.301190</td>\n",
       "      <td>0.092078</td>\n",
       "      <td>-0.099822</td>\n",
       "      <td>0.275679</td>\n",
       "      <td>-0.020586</td>\n",
       "      <td>-0.048048</td>\n",
       "      <td>...</td>\n",
       "      <td>0.053257</td>\n",
       "      <td>0.383810</td>\n",
       "      <td>0.065650</td>\n",
       "      <td>0.073290</td>\n",
       "      <td>0.164891</td>\n",
       "      <td>-0.063697</td>\n",
       "      <td>-0.028915</td>\n",
       "      <td>-0.033661</td>\n",
       "      <td>0.083527</td>\n",
       "      <td>-0.065841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8155</th>\n",
       "      <td>109510</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.366495</td>\n",
       "      <td>-0.410334</td>\n",
       "      <td>-0.440745</td>\n",
       "      <td>-0.608958</td>\n",
       "      <td>0.164707</td>\n",
       "      <td>0.193847</td>\n",
       "      <td>-0.536494</td>\n",
       "      <td>-0.048048</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.065479</td>\n",
       "      <td>-0.090837</td>\n",
       "      <td>-0.132787</td>\n",
       "      <td>0.073290</td>\n",
       "      <td>-0.241145</td>\n",
       "      <td>-0.063697</td>\n",
       "      <td>-0.028915</td>\n",
       "      <td>-0.033661</td>\n",
       "      <td>0.083527</td>\n",
       "      <td>-0.348529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154053</th>\n",
       "      <td>278546</td>\n",
       "      <td>0</td>\n",
       "      <td>0.038650</td>\n",
       "      <td>0.016406</td>\n",
       "      <td>0.301190</td>\n",
       "      <td>0.371651</td>\n",
       "      <td>0.075169</td>\n",
       "      <td>0.060654</td>\n",
       "      <td>-0.020586</td>\n",
       "      <td>-0.048048</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.065479</td>\n",
       "      <td>-0.090837</td>\n",
       "      <td>-0.132787</td>\n",
       "      <td>-0.316556</td>\n",
       "      <td>-0.241145</td>\n",
       "      <td>-0.063697</td>\n",
       "      <td>-0.028915</td>\n",
       "      <td>-0.033661</td>\n",
       "      <td>0.083527</td>\n",
       "      <td>-0.348529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300963</th>\n",
       "      <td>448668</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.366495</td>\n",
       "      <td>-0.158446</td>\n",
       "      <td>0.301190</td>\n",
       "      <td>-0.171601</td>\n",
       "      <td>0.075169</td>\n",
       "      <td>-0.057870</td>\n",
       "      <td>-0.020586</td>\n",
       "      <td>-0.048048</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.065479</td>\n",
       "      <td>-0.090837</td>\n",
       "      <td>-0.132787</td>\n",
       "      <td>-0.316556</td>\n",
       "      <td>0.164891</td>\n",
       "      <td>-0.063697</td>\n",
       "      <td>-0.028915</td>\n",
       "      <td>-0.033661</td>\n",
       "      <td>0.083527</td>\n",
       "      <td>-0.348529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269546</th>\n",
       "      <td>412373</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.366495</td>\n",
       "      <td>-0.410334</td>\n",
       "      <td>-0.051704</td>\n",
       "      <td>-0.171601</td>\n",
       "      <td>-0.099822</td>\n",
       "      <td>-0.297834</td>\n",
       "      <td>-0.536494</td>\n",
       "      <td>-0.048048</td>\n",
       "      <td>...</td>\n",
       "      <td>0.053257</td>\n",
       "      <td>-0.090837</td>\n",
       "      <td>0.110022</td>\n",
       "      <td>-0.152116</td>\n",
       "      <td>0.164891</td>\n",
       "      <td>-0.063697</td>\n",
       "      <td>-0.028915</td>\n",
       "      <td>-0.033661</td>\n",
       "      <td>-0.239387</td>\n",
       "      <td>0.084509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298994</th>\n",
       "      <td>446376</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.050233</td>\n",
       "      <td>0.016406</td>\n",
       "      <td>-0.440745</td>\n",
       "      <td>-0.451249</td>\n",
       "      <td>-0.377708</td>\n",
       "      <td>-0.297834</td>\n",
       "      <td>-0.020586</td>\n",
       "      <td>-0.048048</td>\n",
       "      <td>...</td>\n",
       "      <td>0.053257</td>\n",
       "      <td>-0.040815</td>\n",
       "      <td>0.110022</td>\n",
       "      <td>0.073290</td>\n",
       "      <td>-0.241145</td>\n",
       "      <td>-0.063697</td>\n",
       "      <td>-0.028915</td>\n",
       "      <td>-0.033661</td>\n",
       "      <td>0.083527</td>\n",
       "      <td>0.084509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269429</th>\n",
       "      <td>412242</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.050233</td>\n",
       "      <td>0.016406</td>\n",
       "      <td>-0.440745</td>\n",
       "      <td>0.253381</td>\n",
       "      <td>0.075169</td>\n",
       "      <td>0.060654</td>\n",
       "      <td>-0.020586</td>\n",
       "      <td>-0.048048</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.065479</td>\n",
       "      <td>-0.090837</td>\n",
       "      <td>-0.132787</td>\n",
       "      <td>0.073290</td>\n",
       "      <td>0.164891</td>\n",
       "      <td>-0.063697</td>\n",
       "      <td>-0.028915</td>\n",
       "      <td>-0.033661</td>\n",
       "      <td>0.083527</td>\n",
       "      <td>-0.348529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>100020</td>\n",
       "      <td>0</td>\n",
       "      <td>0.268859</td>\n",
       "      <td>0.268275</td>\n",
       "      <td>0.301190</td>\n",
       "      <td>0.253381</td>\n",
       "      <td>-0.099822</td>\n",
       "      <td>-0.057870</td>\n",
       "      <td>-0.020586</td>\n",
       "      <td>0.459100</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.065479</td>\n",
       "      <td>-0.090837</td>\n",
       "      <td>0.110022</td>\n",
       "      <td>0.073290</td>\n",
       "      <td>0.164891</td>\n",
       "      <td>-0.063697</td>\n",
       "      <td>-0.028915</td>\n",
       "      <td>-0.033661</td>\n",
       "      <td>0.083527</td>\n",
       "      <td>0.084509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97169</th>\n",
       "      <td>212804</td>\n",
       "      <td>0</td>\n",
       "      <td>0.038650</td>\n",
       "      <td>0.016406</td>\n",
       "      <td>-0.440745</td>\n",
       "      <td>-0.451249</td>\n",
       "      <td>0.075169</td>\n",
       "      <td>-0.057870</td>\n",
       "      <td>-0.536494</td>\n",
       "      <td>-0.048048</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.065479</td>\n",
       "      <td>-0.090837</td>\n",
       "      <td>0.110022</td>\n",
       "      <td>0.073290</td>\n",
       "      <td>0.164891</td>\n",
       "      <td>-0.063697</td>\n",
       "      <td>-0.028915</td>\n",
       "      <td>-0.033661</td>\n",
       "      <td>0.083527</td>\n",
       "      <td>0.084509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90581</th>\n",
       "      <td>205165</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.050233</td>\n",
       "      <td>-0.043274</td>\n",
       "      <td>0.301190</td>\n",
       "      <td>0.092078</td>\n",
       "      <td>0.075169</td>\n",
       "      <td>0.060654</td>\n",
       "      <td>-0.020586</td>\n",
       "      <td>-0.048048</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.065479</td>\n",
       "      <td>-0.090837</td>\n",
       "      <td>-0.132787</td>\n",
       "      <td>0.073290</td>\n",
       "      <td>0.164891</td>\n",
       "      <td>-0.063697</td>\n",
       "      <td>-0.028915</td>\n",
       "      <td>-0.033661</td>\n",
       "      <td>0.083527</td>\n",
       "      <td>-0.348529</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>215257 rows × 79 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        SK_ID_CURR  TARGET  AMT_GOODS_PRICE  REGION_POPULATION_RELATIVE  \\\n",
       "125406      245429       0         0.610118                    0.016406   \n",
       "8155        109510       0        -0.366495                   -0.410334   \n",
       "154053      278546       0         0.038650                    0.016406   \n",
       "300963      448668       0        -0.366495                   -0.158446   \n",
       "269546      412373       0        -0.366495                   -0.410334   \n",
       "...            ...     ...              ...                         ...   \n",
       "298994      446376       0        -0.050233                    0.016406   \n",
       "269429      412242       0        -0.050233                    0.016406   \n",
       "16          100020       0         0.268859                    0.268275   \n",
       "97169       212804       0         0.038650                    0.016406   \n",
       "90581       205165       0        -0.050233                   -0.043274   \n",
       "\n",
       "        DAYS_BIRTH  DAYS_EMPLOYED  DAYS_REGISTRATION  DAYS_ID_PUBLISH  \\\n",
       "125406    0.301190       0.092078          -0.099822         0.275679   \n",
       "8155     -0.440745      -0.608958           0.164707         0.193847   \n",
       "154053    0.301190       0.371651           0.075169         0.060654   \n",
       "300963    0.301190      -0.171601           0.075169        -0.057870   \n",
       "269546   -0.051704      -0.171601          -0.099822        -0.297834   \n",
       "...            ...            ...                ...              ...   \n",
       "298994   -0.440745      -0.451249          -0.377708        -0.297834   \n",
       "269429   -0.440745       0.253381           0.075169         0.060654   \n",
       "16        0.301190       0.253381          -0.099822        -0.057870   \n",
       "97169    -0.440745      -0.451249           0.075169        -0.057870   \n",
       "90581     0.301190       0.092078           0.075169         0.060654   \n",
       "\n",
       "        REGION_RATING_CLIENT_W_CITY  REG_CITY_NOT_LIVE_CITY  ...  \\\n",
       "125406                    -0.020586               -0.048048  ...   \n",
       "8155                      -0.536494               -0.048048  ...   \n",
       "154053                    -0.020586               -0.048048  ...   \n",
       "300963                    -0.020586               -0.048048  ...   \n",
       "269546                    -0.536494               -0.048048  ...   \n",
       "...                             ...                     ...  ...   \n",
       "298994                    -0.020586               -0.048048  ...   \n",
       "269429                    -0.020586               -0.048048  ...   \n",
       "16                        -0.020586                0.459100  ...   \n",
       "97169                     -0.536494               -0.048048  ...   \n",
       "90581                     -0.020586               -0.048048  ...   \n",
       "\n",
       "        p_NAME_SELLER_INDUSTRY_Connectivity  p_NAME_YIELD_GROUP_XNA  \\\n",
       "125406                             0.053257                0.383810   \n",
       "8155                              -0.065479               -0.090837   \n",
       "154053                            -0.065479               -0.090837   \n",
       "300963                            -0.065479               -0.090837   \n",
       "269546                             0.053257               -0.090837   \n",
       "...                                     ...                     ...   \n",
       "298994                             0.053257               -0.040815   \n",
       "269429                            -0.065479               -0.090837   \n",
       "16                                -0.065479               -0.090837   \n",
       "97169                             -0.065479               -0.090837   \n",
       "90581                             -0.065479               -0.090837   \n",
       "\n",
       "        p_NAME_YIELD_GROUP_high  p_NAME_YIELD_GROUP_low_action  \\\n",
       "125406                 0.065650                       0.073290   \n",
       "8155                  -0.132787                       0.073290   \n",
       "154053                -0.132787                      -0.316556   \n",
       "300963                -0.132787                      -0.316556   \n",
       "269546                 0.110022                      -0.152116   \n",
       "...                         ...                            ...   \n",
       "298994                 0.110022                       0.073290   \n",
       "269429                -0.132787                       0.073290   \n",
       "16                     0.110022                       0.073290   \n",
       "97169                  0.110022                       0.073290   \n",
       "90581                 -0.132787                       0.073290   \n",
       "\n",
       "        p_NAME_YIELD_GROUP_low_normal  p_PRODUCT_COMBINATION_Card Street  \\\n",
       "125406                       0.164891                          -0.063697   \n",
       "8155                        -0.241145                          -0.063697   \n",
       "154053                      -0.241145                          -0.063697   \n",
       "300963                       0.164891                          -0.063697   \n",
       "269546                       0.164891                          -0.063697   \n",
       "...                               ...                                ...   \n",
       "298994                      -0.241145                          -0.063697   \n",
       "269429                       0.164891                          -0.063697   \n",
       "16                           0.164891                          -0.063697   \n",
       "97169                        0.164891                          -0.063697   \n",
       "90581                        0.164891                          -0.063697   \n",
       "\n",
       "        p_PRODUCT_COMBINATION_Cash Street: high  \\\n",
       "125406                                -0.028915   \n",
       "8155                                  -0.028915   \n",
       "154053                                -0.028915   \n",
       "300963                                -0.028915   \n",
       "269546                                -0.028915   \n",
       "...                                         ...   \n",
       "298994                                -0.028915   \n",
       "269429                                -0.028915   \n",
       "16                                    -0.028915   \n",
       "97169                                 -0.028915   \n",
       "90581                                 -0.028915   \n",
       "\n",
       "        p_PRODUCT_COMBINATION_Cash X-Sell: high  \\\n",
       "125406                                -0.033661   \n",
       "8155                                  -0.033661   \n",
       "154053                                -0.033661   \n",
       "300963                                -0.033661   \n",
       "269546                                -0.033661   \n",
       "...                                         ...   \n",
       "298994                                -0.033661   \n",
       "269429                                -0.033661   \n",
       "16                                    -0.033661   \n",
       "97169                                 -0.033661   \n",
       "90581                                 -0.033661   \n",
       "\n",
       "        p_PRODUCT_COMBINATION_Cash X-Sell: low  \\\n",
       "125406                                0.083527   \n",
       "8155                                  0.083527   \n",
       "154053                                0.083527   \n",
       "300963                                0.083527   \n",
       "269546                               -0.239387   \n",
       "...                                        ...   \n",
       "298994                                0.083527   \n",
       "269429                                0.083527   \n",
       "16                                    0.083527   \n",
       "97169                                 0.083527   \n",
       "90581                                 0.083527   \n",
       "\n",
       "        p_PRODUCT_COMBINATION_POS industry with interest  \n",
       "125406                                         -0.065841  \n",
       "8155                                           -0.348529  \n",
       "154053                                         -0.348529  \n",
       "300963                                         -0.348529  \n",
       "269546                                          0.084509  \n",
       "...                                                  ...  \n",
       "298994                                          0.084509  \n",
       "269429                                         -0.348529  \n",
       "16                                              0.084509  \n",
       "97169                                           0.084509  \n",
       "90581                                          -0.348529  \n",
       "\n",
       "[215257 rows x 79 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd \n",
    "import joblib\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from boruta import BorutaPy\n",
    "#加载数据\n",
    "pd_data = joblib.load('data/train_woe.pkl')\n",
    "pd_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "asian-check",
   "metadata": {},
   "outputs": [],
   "source": [
    "#处理数据，去掉id 和 目标值\n",
    "pd_x = pd_data.drop(['SK_ID_CURR', 'TARGET'], axis=1)\n",
    "x = pd_x.values   # 特征\n",
    "y = pd_data[['TARGET']].values # 目标\n",
    "y = y.ravel() # 将多维数组降位一维 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "global-religion",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TARGET\n",
       "0         197840\n",
       "1          17417\n",
       "dtype: int64"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd_data[['TARGET']].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "harmful-marriage",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BorutaPy(estimator=RandomForestClassifier(class_weight='balanced', max_depth=5,\n",
       "                                          n_estimators=248, n_jobs=-1,\n",
       "                                          random_state=RandomState(MT19937) at 0x7FA14A4B4D40),\n",
       "         max_iter=10, n_estimators='auto',\n",
       "         random_state=RandomState(MT19937) at 0x7FA14A4B4D40)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 先定义一个随机森林分类器\n",
    "rf = RandomForestClassifier(n_jobs=-1, class_weight='balanced', max_depth=5)\n",
    "'''\n",
    "BorutaPy function\n",
    "estimator : 所使用的分类器\n",
    "n_estimators : 分类器数量, 默认值 = 1000\n",
    "max_iter : 最大迭代次数, 默认值 = 100\n",
    "'''\n",
    "feat_selector = BorutaPy(rf, n_estimators='auto', random_state=1, max_iter=10)\n",
    "feat_selector.fit(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "favorite-marsh",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "       False,  True,  True,  True,  True,  True, False,  True,  True,\n",
       "        True, False, False,  True,  True,  True, False,  True,  True,\n",
       "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "        True, False,  True,  True,  True,  True, False,  True,  True,\n",
       "        True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "        True,  True,  True,  True,  True])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feat_selector.support_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "basic-agency",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>selected</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AMT_GOODS_PRICE</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>REGION_POPULATION_RELATIVE</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DAYS_BIRTH</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DAYS_EMPLOYED</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DAYS_REGISTRATION</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>p_PRODUCT_COMBINATION_Card Street</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>p_PRODUCT_COMBINATION_Cash Street: high</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>p_PRODUCT_COMBINATION_Cash X-Sell: high</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>p_PRODUCT_COMBINATION_Cash X-Sell: low</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>p_PRODUCT_COMBINATION_POS industry with interest</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>77 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             feature  selected\n",
       "0                                    AMT_GOODS_PRICE      True\n",
       "1                         REGION_POPULATION_RELATIVE      True\n",
       "2                                         DAYS_BIRTH      True\n",
       "3                                      DAYS_EMPLOYED      True\n",
       "4                                  DAYS_REGISTRATION      True\n",
       "..                                               ...       ...\n",
       "72                 p_PRODUCT_COMBINATION_Card Street      True\n",
       "73           p_PRODUCT_COMBINATION_Cash Street: high      True\n",
       "74           p_PRODUCT_COMBINATION_Cash X-Sell: high      True\n",
       "75            p_PRODUCT_COMBINATION_Cash X-Sell: low      True\n",
       "76  p_PRODUCT_COMBINATION_POS industry with interest      True\n",
       "\n",
       "[77 rows x 2 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dic_ft_select = dict()\n",
    "# feat_selector.support_ # 返回特征是否有用，false可以去掉\n",
    "for ft, seleted in zip(pd_x.columns.to_list(), feat_selector.support_):\n",
    "    dic_ft_select[ft] = seleted\n",
    "pd_ft_select = pd.DataFrame({'feature':pd_x.columns.to_list(), \"selected\": feat_selector.support_})\n",
    "pd_ft_select"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "alert-bhutan",
   "metadata": {},
   "source": [
    "- 方差膨胀系数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "indie-police",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd \n",
    "import joblib\n",
    "\n",
    "from statsmodels.stats.outliers_influence import variance_inflation_factor\n",
    "pd_data = joblib.load('./data/train_woe.pkl')\n",
    "#去掉ID和目标值\n",
    "pd_x = pd_data.drop(['SK_ID_CURR', 'TARGET'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "spectacular-heater",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/vincent/.local/lib/python3.8/site-packages/statsmodels/stats/outliers_influence.py:193: RuntimeWarning: divide by zero encountered in double_scalars\n",
      "  vif = 1. / (1. - r_squared_i)\n",
      "/Users/vincent/.local/lib/python3.8/site-packages/statsmodels/regression/linear_model.py:1717: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  return 1 - self.ssr/self.uncentered_tss\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inf\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>VIF</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AMT_GOODS_PRICE</td>\n",
       "      <td>1.181193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>REGION_POPULATION_RELATIVE</td>\n",
       "      <td>1.848021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DAYS_BIRTH</td>\n",
       "      <td>3.322928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>DAYS_EMPLOYED</td>\n",
       "      <td>1.689302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DAYS_REGISTRATION</td>\n",
       "      <td>1.182928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>p_PRODUCT_COMBINATION_Card Street</td>\n",
       "      <td>2.282718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>p_PRODUCT_COMBINATION_Cash Street: high</td>\n",
       "      <td>2.392276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>p_PRODUCT_COMBINATION_Cash X-Sell: high</td>\n",
       "      <td>1.933213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>p_PRODUCT_COMBINATION_Cash X-Sell: low</td>\n",
       "      <td>2.112973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>p_PRODUCT_COMBINATION_POS industry with interest</td>\n",
       "      <td>2.045593</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>77 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             feature       VIF\n",
       "0                                    AMT_GOODS_PRICE  1.181193\n",
       "1                         REGION_POPULATION_RELATIVE  1.848021\n",
       "2                                         DAYS_BIRTH  3.322928\n",
       "3                                      DAYS_EMPLOYED  1.689302\n",
       "4                                  DAYS_REGISTRATION  1.182928\n",
       "..                                               ...       ...\n",
       "72                 p_PRODUCT_COMBINATION_Card Street  2.282718\n",
       "73           p_PRODUCT_COMBINATION_Cash Street: high  2.392276\n",
       "74           p_PRODUCT_COMBINATION_Cash X-Sell: high  1.933213\n",
       "75            p_PRODUCT_COMBINATION_Cash X-Sell: low  2.112973\n",
       "76  p_PRODUCT_COMBINATION_POS industry with interest  2.045593\n",
       "\n",
       "[77 rows x 2 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#定义计算函数\n",
    "def checkVIF_new(df):\n",
    "    lst_col = df.columns\n",
    "    x = np.matrix(df)\n",
    "    VIF_list = [variance_inflation_factor(x,i) for i in range(x.shape[1])]\n",
    "    VIF = pd.DataFrame({'feature':lst_col,\"VIF\":VIF_list})\n",
    "    max_VIF = max(VIF_list)\n",
    "    print(max_VIF)\n",
    "    return VIF\n",
    "df_vif = checkVIF_new(pd_x)\n",
    "df_vif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "accessible-drama",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>VIF</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>DAYS_BIRTH</td>\n",
       "      <td>3.322928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>YEARS_BEGINEXPLUATATION_AVG</td>\n",
       "      <td>4.558895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>FLOORSMAX_MEDI</td>\n",
       "      <td>5.452477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>TOTALAREA_MODE</td>\n",
       "      <td>5.242828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>AMT_REQ_CREDIT_BUREAU_YEAR</td>\n",
       "      <td>4.186162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>NAME_INCOME_TYPE_Pensioner</td>\n",
       "      <td>3.437966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>EMERGENCYSTATE_MODE_No</td>\n",
       "      <td>3.851456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>b_CREDIT_DAY_OVERDUE</td>\n",
       "      <td>inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>b_AMT_CREDIT_SUM_OVERDUE</td>\n",
       "      <td>inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>b_CREDIT_TYPE_Car loan</td>\n",
       "      <td>3.136588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>b_CREDIT_TYPE_Mortgage</td>\n",
       "      <td>inf</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>p_NAME_PORTFOLIO_POS</td>\n",
       "      <td>3.288162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>p_NAME_YIELD_GROUP_XNA</td>\n",
       "      <td>4.261597</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        feature       VIF\n",
       "2                    DAYS_BIRTH  3.322928\n",
       "11  YEARS_BEGINEXPLUATATION_AVG  4.558895\n",
       "12               FLOORSMAX_MEDI  5.452477\n",
       "13               TOTALAREA_MODE  5.242828\n",
       "16   AMT_REQ_CREDIT_BUREAU_YEAR  4.186162\n",
       "18   NAME_INCOME_TYPE_Pensioner  3.437966\n",
       "23       EMERGENCYSTATE_MODE_No  3.851456\n",
       "27         b_CREDIT_DAY_OVERDUE       inf\n",
       "33     b_AMT_CREDIT_SUM_OVERDUE       inf\n",
       "35       b_CREDIT_TYPE_Car loan  3.136588\n",
       "38       b_CREDIT_TYPE_Mortgage       inf\n",
       "65         p_NAME_PORTFOLIO_POS  3.288162\n",
       "68       p_NAME_YIELD_GROUP_XNA  4.261597"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_vif[df_vif['VIF'] > 3] "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "verbal-nursing",
   "metadata": {},
   "source": [
    "- RFE 递归特征消除"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "sublime-concert",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>DAYS_EMPLOYED</th>\n",
       "      <th>EXT_SOURCE_2</th>\n",
       "      <th>EXT_SOURCE_3</th>\n",
       "      <th>PAYMENT_RATE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>125406</th>\n",
       "      <td>245429</td>\n",
       "      <td>0</td>\n",
       "      <td>0.092078</td>\n",
       "      <td>-0.194908</td>\n",
       "      <td>0.843085</td>\n",
       "      <td>0.441912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8155</th>\n",
       "      <td>109510</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.608958</td>\n",
       "      <td>-1.045357</td>\n",
       "      <td>-0.756028</td>\n",
       "      <td>0.103977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154053</th>\n",
       "      <td>278546</td>\n",
       "      <td>0</td>\n",
       "      <td>0.371651</td>\n",
       "      <td>0.357743</td>\n",
       "      <td>-0.057383</td>\n",
       "      <td>0.441912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300963</th>\n",
       "      <td>448668</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.171601</td>\n",
       "      <td>0.951342</td>\n",
       "      <td>-0.057383</td>\n",
       "      <td>0.441912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269546</th>\n",
       "      <td>412373</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.171601</td>\n",
       "      <td>-0.194908</td>\n",
       "      <td>-0.756028</td>\n",
       "      <td>-0.259090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298994</th>\n",
       "      <td>446376</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.451249</td>\n",
       "      <td>-1.045357</td>\n",
       "      <td>-0.756028</td>\n",
       "      <td>0.103977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269429</th>\n",
       "      <td>412242</td>\n",
       "      <td>0</td>\n",
       "      <td>0.253381</td>\n",
       "      <td>0.357743</td>\n",
       "      <td>-0.057383</td>\n",
       "      <td>-0.259090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>100020</td>\n",
       "      <td>0</td>\n",
       "      <td>0.253381</td>\n",
       "      <td>0.357743</td>\n",
       "      <td>0.843085</td>\n",
       "      <td>0.103977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97169</th>\n",
       "      <td>212804</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.451249</td>\n",
       "      <td>0.013503</td>\n",
       "      <td>-0.756028</td>\n",
       "      <td>-0.259090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90581</th>\n",
       "      <td>205165</td>\n",
       "      <td>0</td>\n",
       "      <td>0.092078</td>\n",
       "      <td>-0.533934</td>\n",
       "      <td>-0.756028</td>\n",
       "      <td>-0.259090</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>215257 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        SK_ID_CURR  TARGET  DAYS_EMPLOYED  EXT_SOURCE_2  EXT_SOURCE_3  \\\n",
       "125406      245429       0       0.092078     -0.194908      0.843085   \n",
       "8155        109510       0      -0.608958     -1.045357     -0.756028   \n",
       "154053      278546       0       0.371651      0.357743     -0.057383   \n",
       "300963      448668       0      -0.171601      0.951342     -0.057383   \n",
       "269546      412373       0      -0.171601     -0.194908     -0.756028   \n",
       "...            ...     ...            ...           ...           ...   \n",
       "298994      446376       0      -0.451249     -1.045357     -0.756028   \n",
       "269429      412242       0       0.253381      0.357743     -0.057383   \n",
       "16          100020       0       0.253381      0.357743      0.843085   \n",
       "97169       212804       0      -0.451249      0.013503     -0.756028   \n",
       "90581       205165       0       0.092078     -0.533934     -0.756028   \n",
       "\n",
       "        PAYMENT_RATE  \n",
       "125406      0.441912  \n",
       "8155        0.103977  \n",
       "154053      0.441912  \n",
       "300963      0.441912  \n",
       "269546     -0.259090  \n",
       "...              ...  \n",
       "298994      0.103977  \n",
       "269429     -0.259090  \n",
       "16          0.103977  \n",
       "97169      -0.259090  \n",
       "90581      -0.259090  \n",
       "\n",
       "[215257 rows x 6 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd \n",
    "import joblib\n",
    "\n",
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.svm import SVR\n",
    "pd_data = joblib.load('data/final_data.pkl')\n",
    "pd_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "lovely-extra",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd_x = pd_data.drop(['SK_ID_CURR', 'TARGET'], axis=1)\n",
    "x = pd_x.values\n",
    "y = pd_data[['TARGET']].values\n",
    "y = y.ravel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "regular-titanium",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>DAYS_EMPLOYED</th>\n",
       "      <th>EXT_SOURCE_2</th>\n",
       "      <th>EXT_SOURCE_3</th>\n",
       "      <th>PAYMENT_RATE</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>125406</th>\n",
       "      <td>0.092078</td>\n",
       "      <td>-0.194908</td>\n",
       "      <td>0.843085</td>\n",
       "      <td>0.441912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8155</th>\n",
       "      <td>-0.608958</td>\n",
       "      <td>-1.045357</td>\n",
       "      <td>-0.756028</td>\n",
       "      <td>0.103977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154053</th>\n",
       "      <td>0.371651</td>\n",
       "      <td>0.357743</td>\n",
       "      <td>-0.057383</td>\n",
       "      <td>0.441912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300963</th>\n",
       "      <td>-0.171601</td>\n",
       "      <td>0.951342</td>\n",
       "      <td>-0.057383</td>\n",
       "      <td>0.441912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269546</th>\n",
       "      <td>-0.171601</td>\n",
       "      <td>-0.194908</td>\n",
       "      <td>-0.756028</td>\n",
       "      <td>-0.259090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>298994</th>\n",
       "      <td>-0.451249</td>\n",
       "      <td>-1.045357</td>\n",
       "      <td>-0.756028</td>\n",
       "      <td>0.103977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269429</th>\n",
       "      <td>0.253381</td>\n",
       "      <td>0.357743</td>\n",
       "      <td>-0.057383</td>\n",
       "      <td>-0.259090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.253381</td>\n",
       "      <td>0.357743</td>\n",
       "      <td>0.843085</td>\n",
       "      <td>0.103977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97169</th>\n",
       "      <td>-0.451249</td>\n",
       "      <td>0.013503</td>\n",
       "      <td>-0.756028</td>\n",
       "      <td>-0.259090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90581</th>\n",
       "      <td>0.092078</td>\n",
       "      <td>-0.533934</td>\n",
       "      <td>-0.756028</td>\n",
       "      <td>-0.259090</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>215257 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        DAYS_EMPLOYED  EXT_SOURCE_2  EXT_SOURCE_3  PAYMENT_RATE\n",
       "125406       0.092078     -0.194908      0.843085      0.441912\n",
       "8155        -0.608958     -1.045357     -0.756028      0.103977\n",
       "154053       0.371651      0.357743     -0.057383      0.441912\n",
       "300963      -0.171601      0.951342     -0.057383      0.441912\n",
       "269546      -0.171601     -0.194908     -0.756028     -0.259090\n",
       "...               ...           ...           ...           ...\n",
       "298994      -0.451249     -1.045357     -0.756028      0.103977\n",
       "269429       0.253381      0.357743     -0.057383     -0.259090\n",
       "16           0.253381      0.357743      0.843085      0.103977\n",
       "97169       -0.451249      0.013503     -0.756028     -0.259090\n",
       "90581        0.092078     -0.533934     -0.756028     -0.259090\n",
       "\n",
       "[215257 rows x 4 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "minimal-luxembourg",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/vincent/.local/lib/python3.8/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass n_features_to_select=3 as keyword args. From version 1.0 (renaming of 0.25) passing these as positional arguments will result in an error\n",
      "  warnings.warn(f\"Pass {args_msg} as keyword args. From version \"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>selected</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>DAYS_EMPLOYED</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>EXT_SOURCE_2</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>EXT_SOURCE_3</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PAYMENT_RATE</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         feature  selected\n",
       "0  DAYS_EMPLOYED     False\n",
       "1   EXT_SOURCE_2      True\n",
       "2   EXT_SOURCE_3      True\n",
       "3   PAYMENT_RATE      True"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#定义分类器\n",
    "estimator = SVR(kernel=\"linear\")\n",
    "selector = RFE(estimator, 3, step=1) # step 一次去掉几个特征\n",
    "selector = selector.fit(x, y)\n",
    "#展示选择参数\n",
    "dic_ft_select = dict()\n",
    "for ft, seleted in zip(pd_x.columns.to_list(), selector.support_):\n",
    "    dic_ft_select[ft] = seleted\n",
    "pd_ft_select = pd.DataFrame({'feature':pd_x.columns.to_list(), \"selected\": selector.support_})\n",
    "pd_ft_select"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sensitive-database",
   "metadata": {},
   "source": [
    "- 基于L1的特征选择"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "subjective-counter",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150, 4)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.datasets import load_iris\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "iris = load_iris()\n",
    "X, y = iris.data, iris.target\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "interior-hammer",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150, 3)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lsvc = LinearSVC(C=0.01, penalty=\"l1\", dual=False).fit(X, y)\n",
    "model = SelectFromModel(lsvc, prefit=True)\n",
    "X_new = model.transform(X)\n",
    "X_new.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "turned-integer",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
